# 
# Description:
# This script constructs Table A4
# for Appendix A. First the news
# data is collected.
#
# Requires: terror.dta, motives.xlsx, terror_new.csv, vanderbilt.RDS
#
# Author: Sean Nossek
#
# Date: August 04, 2020

# load packages

if (!require("pacman")) install.packages("pacman")

#lfe version manage
install.packages("lfe")
library(lfe)

pacman::p_load(tidyverse, readstata13, lubridate, haven, readxl, lubridate, cdlTools, stringi, stargazer, maps)

# set dir 
setwd("C:/Users/Sean/Dropbox/Eran/Terrorism_Left/Paper/Revisions/Revised Paper/Replication/Paper Data")


# read terror event data and merge with manually coded motives
terror_old <- read.dta13("terror.dta") %>%
  mutate(fips_1 = str_pad(FIPSCounty,5,pad = "0")) %>%
  arrange(year, fips_1)
motives <- read_excel("motives.xlsx")
terror_new <- read_csv("terror_new.csv") %>%
  mutate(fips_1 = str_pad(FIPSCounty,5,pad = "0"))
terror_base <- rbind(terror_old, terror_new)
terror_base <- merge(terror_base, motives, by = "eventid", all.x = TRUE)

terror_base$weaptype1_txt[terror_base$weaptype1_txt == "Explosives/Bombs/Dynamite"] <- "Explosives"

terror_base$state_abbv <- fips(terror_base$state_fips, to = "Abbreviation")

terror_base$state_abbv_1 <- str_pad(terror_base$state_abbv, 3, "left", pad = " ")

# clean state abbreviations

stri_sub(terror_base$state_abbv,2,1) <- "."

stri_sub(terror_base$state_abbv,4,4) <- "."

# select vars from terror for news analysis

new_news <- terror_base %>%
  select(eventid, year, imonth, iday, provstate, FIPSCounty, city, success, location, summary, claimed, attacktype1_txt, weaptype1_txt, corp1, target1, corp2, target2, gname, motive, motive1, motive2, nkill, nwound, INT_LOG, propcomment, property, propextent, addnotes, scite1, scite2, scite3, state_abbv, state_abbv_1, doubtterr) %>%
  mutate(provstate = str_remove_all(provstate, "\\(.*\\)"),
         city_state = paste(city, provstate, sep = ", "),
         imonth = str_pad(imonth, 2, pad = "0"), 
         iday = str_pad(iday, 2, pad = "0"),
         date = paste(imonth, iday, sep = "-"),
         date = paste(date, year, sep = "-"),
         date_1 = as.Date(date, format = "%m-%d-%Y"),
         gname = str_remove(gname, "\\(.*\\)"), 
         gname = ifelse(gname == "", NA, gname), 
         corp1 = ifelse(corp1 == "", NA, corp1),
         target1 = ifelse(target1 == "", NA, target1),
         weaptype1_txt = ifelse(weaptype1_txt == "", NA, weaptype1_txt),
         motive = ifelse(motive == "", NA, motive),
         location = ifelse(location == "", NA, target1))

# read vanderbilt news data
vandy <- readRDS("vanderbilt.RDS")

# clean
vandy <- vandy %>%
  mutate(date_1 = str_remove(date, "[A-Z][a-z]*,"),
         month = str_extract(date_1, "[A-Z][a-z][a-z]"),
         day = str_extract(date_1, " [0-9][0-9],"), 
         day = str_remove(day, ","), 
         year = str_extract(date_1, " [0-9][0-9][0-9][0-9]"),
         month_n = case_when(month == "Jan" ~ "01",
                             month == "Feb" ~ "02",
                             month == "Mar" ~ "03",
                             month == "Apr" ~ "04",
                             month == "May" ~ "05",
                             month == "Jun" ~ "06",
                             month == "Jul" ~ "07",
                             month == "Aug" ~ "08",
                             month == "Sep" ~ "09",
                             month == "Oct" ~ "10",
                             month == "Nov" ~ "11",
                             month == "Dec" ~ "12"))

vandy <- vandy %>% 
  mutate( date_1 = paste(month_n, day, sep = "-"), 
          date_1 = paste(date_1, year, sep = "-"),
          date_1 = as.Date(date_1, format = "%m-%d-%Y"))

new_news$total_stories <- NA
new_news$cbs_n <- NA
new_news$abc_n <- NA
new_news$nbc_n <- NA
new_news$cbs_duration_sec <- NA
new_news$abc_duration_sec <- NA
new_news$nbc_duration_sec <- NA

vandy <- vandy %>%
  filter(program_title %in% c( "ABC Evening News", "NBC Evening News", "CBS Evening News", "ABC Special", "CBS Special", "NBC Special")) %>%
  mutate(program = case_when(program_title ==  "ABC Evening News" ~ "ABC",
                             program_title ==  "NBC Evening News" ~ "NBC",
                             program_title ==  "CBS Evening News" ~ "CBS",
                             program_title ==  "ABC Special" ~ "ABC",
                             program_title ==  "CBS Special" ~ "CBS",
                             program_title ==  "NBC Special" ~ "NBC"))


# loop over attacks to select stories during time period and referring to place of attack 
for (i in 1:nrow(new_news)) {
  
  print(i)
  
  vandy_sub <- NA
  
  vandy_sub <- vandy %>%
    filter(date_1  >= new_news$date_1[i] & date_1  <= new_news$date_1[i] + 11,) 
  
  vandy_sub <- vandy_sub %>% filter(grepl(tolower(new_news$city[i]), tolower(broadcast_abstract)) | grepl(tolower(new_news$city[i]), tolower(broadcast_title)))
  
  vandy_sub <- vandy_sub %>% filter(grepl(new_news$provstate[i], broadcast_title) |  grepl(new_news$provstate[i], broadcast_abstract) | grepl(new_news$state_abbv[i], broadcast_title, fixed = TRUE) | grepl(new_news$state_abbv[i], broadcast_abstract, fixed = TRUE) |  grepl(new_news$state_abbv_1[i], broadcast_title, fixed = TRUE) | grepl(new_news$state_abbv_1[i], broadcast_abstract, fixed = TRUE))
  
  new_news$total_stories[i] <- nrow(vandy_sub)
  
  if(nrow(vandy_sub) == 0){
    new_news$cbs_n[i] <- 0
    new_news$abc_n[i] <- 0
    new_news$nbc_n[i] <- 0
    new_news$cbs_duration_sec[i] <- 0
    new_news$abc_duration_sec[i] <- 0
    new_news$nbc_duration_sec[i] <- 0
    next
  }
  
  # filter with keywords for attack coverage
  vandy_sub <- vandy_sub %>%
    filter(grepl("terror|attack|shoot|bomb|explosi|target| gun| arm|perpetrat|weapon|arson| shot| casualt| kill| injur|assailant| fire|incind", tolower(broadcast_abstract)) | 
             grepl("terror|attack|shoot|bomb|explosi|target| gun| arm|perpetrat|weapon|arson| shot| casualt| kill| injur|assailant| fire|incind", tolower(broadcast_title)) |
             agrepl(tolower(new_news$gname[i]), tolower(broadcast_abstract)) | 
             agrepl(tolower(new_news$gname[i]), tolower(broadcast_title)) | 
             agrepl(tolower(new_news$corp1[i]), tolower(broadcast_title)) | 
             agrepl(tolower(new_news$corp1[i]), tolower(broadcast_abstract)) | 
             agrepl(tolower(new_news$target1[i]), tolower(broadcast_title))  |
             agrepl(tolower(new_news$target1[i]), tolower(broadcast_abstract)) |
             agrepl(tolower(new_news$weaptype1_txt[i]), tolower(broadcast_abstract)) |
             agrepl(tolower(new_news$motive[i]), tolower(broadcast_abstract)) |
             agrepl(tolower(new_news$location[i]), tolower(broadcast_title))  |
             agrepl(tolower(new_news$location[i]), tolower(broadcast_abstract))) 
  
  new_news$cbs_n[i] <- sum(vandy_sub$program == "CBS")
  new_news$abc_n[i] <- sum(vandy_sub$program == "ABC")
  new_news$nbc_n[i] <- sum(vandy_sub$program == "NBC")
  new_news$cbs_duration_sec[i] <- sum(vandy_sub$broadcast_duration[vandy_sub$program == "CBS"])
  new_news$abc_duration_sec[i] <- sum(vandy_sub$broadcast_duration[vandy_sub$program == "ABC"])
  new_news$nbc_duration_sec[i] <- sum(vandy_sub$broadcast_duration[vandy_sub$program == "NBC"])
  
  
}

# create variables for analysis
new_news$terror_broadcasts <- new_news$cbs_n + new_news$abc_n + new_news$nbc_n

new_news$nonterror_broadcasts <- new_news$total_stories - new_news$terror_broadcasts

new_news$terror_story_length <- new_news$cbs_duration_sec + new_news$abc_duration_sec + new_news$nbc_duration_sec

new_news$abb <- substr(new_news$state_abbv_1,2,3)

# merge regions
regions <- maps::state.fips %>%
  select(abb, region)

new_news <- new_news %>%
  left_join(regions, by = "abb") %>%
  mutate(region_year = paste(region, year, sep = "-"))

new_news <- new_news %>%
  distinct()

# Table A4

mod1 <- felm(terror_broadcasts ~ success + nkill + nonterror_broadcasts | FIPSCounty + year + imonth + motive1 + weaptype1_txt + attacktype1_txt + region_year | 0 | FIPSCounty, data = new_news)
mod2 <- felm(cbs_n ~ success + nkill + nonterror_broadcasts | FIPSCounty + year + imonth + motive1 + weaptype1_txt + attacktype1_txt + region_year | 0 | FIPSCounty, data = new_news)
mod3 <- felm(abc_n ~ success + nkill + nonterror_broadcasts | FIPSCounty + year + imonth + motive1 + weaptype1_txt + attacktype1_txt + region_year | 0 | FIPSCounty, data = new_news)
mod4 <- felm(nbc_n ~ success + nkill + nonterror_broadcasts | FIPSCounty + year + imonth + motive1 + weaptype1_txt + attacktype1_txt + region_year | 0 | FIPSCounty, data = new_news)

stargazer(mod1, mod2, mod3, mod4, type = "latex", star.char = c("*", "**", "***"),
          star.cutoffs = c(.05, .01, .001))

mod1 <- felm(terror_story_length ~ success + nkill + nonterror_broadcasts | FIPSCounty + year + imonth + motive1 + weaptype1_txt + attacktype1_txt + region_year | 0 | FIPSCounty, data = new_news)
mod2 <- felm(cbs_duration_sec ~ success + nkill + nonterror_broadcasts | FIPSCounty + year + imonth + motive1 + weaptype1_txt + attacktype1_txt + region_year | 0 | FIPSCounty, data = new_news)
mod3 <- felm(abc_duration_sec ~ success + nkill + nonterror_broadcasts | FIPSCounty + year + imonth + motive1 + weaptype1_txt + attacktype1_txt + region_year | 0 | FIPSCounty, data = new_news)
mod4 <- felm(nbc_duration_sec ~ success + nkill + nonterror_broadcasts | FIPSCounty + year + imonth + motive1 + weaptype1_txt + attacktype1_txt + region_year | 0 | FIPSCounty, data = new_news)

stargazer(mod1, mod2, mod3, mod4, type = "latex", star.char = c("*", "**", "***"),
          star.cutoffs = c(.05, .01, .001))
